/*
 * Copyright 2009 by Konstantin Bulenkov
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.bulenkov.crawler.cmd;

import com.bulenkov.crawler.CURL;
import com.bulenkov.crawler.MultiThreadCrawlerJob;
import com.bulenkov.crawler.event.CrawlerListener;
import com.bulenkov.crawler.filter.Filter;
import com.bulenkov.crawler.filter.RegexFilter;
import com.bulenkov.crawler.model.LinkWiderCrawlerModel;

import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;

/**
 * @author Konstantin Bulenkov (konstantin@bulenkov.com)
 */
public class LentaRuJob {
  public static void main(String[] args) {
    MultiThreadCrawlerJob job = new MultiThreadCrawlerJob("Lenta.Ru", 40);
    LinkWiderCrawlerModel model = new LinkWiderCrawlerModel(
        "http://lenta.ru/news/2005/",
        "http://lenta.ru/news/2006/",
        "http://lenta.ru/news/2007/",
        "http://lenta.ru/news/2008/",
        "http://lenta.ru/news/2009/");
    RegexFilter lenta = new RegexFilter("^http://lenta\\.ru(|/.*)$");
    Filter stopFilter = new RegexFilter("^http://.*lenta\\.ru/.*/_(P|p)rinted.htm$");
    model.setFilter(lenta);
    model.setStopFilter(stopFilter);
    model.addListener(new CrawlerListener() {
      private Filter filter = new RegexFilter("^http://.*lenta\\.ru/news/[0-9]{4}/[0-9]{2}/[0-9]{2}/.*/_(P|p)rinted.htm$");
      private File out = new File("lenta.txt");
      private FileOutputStream fos;
      public Filter getLinkFilter() {
        return filter;
      }
      public void linkAdded(CURL curl) {
        try {
          if (fos == null) fos = new FileOutputStream(out,true);
          fos.write((curl.url() + "\n").getBytes());
        } catch (FileNotFoundException e) {
          e.printStackTrace();
        } catch (IOException e) {
          e.printStackTrace();
        }
      }
    });
    job.setModel(model);
    job.start();
  }
}
